{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [],
   "source": [
    "import random\n",
    "import selenium\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from bs4 import BeautifulSoup as bf\n",
    "import os\n",
    "import json\n",
    "import time\n",
    "from paras import *\n",
    "#config parameters domin\n",
    "store_path = GET_PLATFORM()\n",
    "\n",
    "def write_json(file_name,obj_dict,store_dir):\n",
    "    json_str = json.dumps(obj_dict,ensure_ascii=False)\n",
    "    file_obsolute_path = os.path.join(store_dir,file_name)\n",
    "    print(\"file : {} write success!\".format(file_obsolute_path))\n",
    "    with open(file_obsolute_path, 'w' , encoding='utf-8') as json_file:\n",
    "        json_file.write(json_str)\n",
    "\n",
    "def parse_action(action):\n",
    "    #parse comment action\n",
    "    action_dic={}\n",
    "    action_split=action.split()\n",
    "    if action_split[1] == \"Android\" or action_split[1] ==\"iPhone\":\n",
    "        action_dic[\"device\"] = action_split[1]\n",
    "        action_dic[\"ding\"] =  action_split[-4][2]\n",
    "        action_dic[\"cai\"] =  action_split[-3][2]\n",
    "    else:#may from PC device\n",
    "        action_dic[\"device\"] = \"PC\"\n",
    "        action_dic[\"ding\"] =  action_split[-4][2]\n",
    "        action_dic[\"cai\"] =  action_split[-3][2]\n",
    "    return action_dic\n",
    "\n",
    "def click_webelement(button):\n",
    "    if isinstance(button,list):\n",
    "        for sub_button in button:\n",
    "            time.sleep(1)\n",
    "            try:\n",
    "                sub_button.click()\n",
    "            except:\n",
    "                print(\"unable to click the button!\")\n",
    "\n",
    "    if isinstance(button,selenium.webdriver.remote.webelement.WebElement):\n",
    "        try:\n",
    "            button.click()\n",
    "        except:\n",
    "            print(\"unable to click the button\")\n",
    "\n",
    "def get_pages(browser):\n",
    "    comment_sum=int(browser.find_element(By.CSS_SELECTOR,\"#panelTitle > em\").text)\n",
    "    per_page_comments=30\n",
    "    pages=comment_sum//per_page_comments+1\n",
    "    return pages\n",
    "\n",
    "def get_browser(url):\n",
    "    browser = webdriver.Chrome()\n",
    "    browser.get(url)\n",
    "    return browser\n",
    "\n",
    "def get_sigle_page_comments_dict(comments_dic,doc_model,commenter_url_list):\n",
    "\n",
    "    time.sleep(random.randint(0,4))\n",
    "    comments=doc_model.select(\"#commentTabBlockNew > ul.comment_listBox > li\")\n",
    "\n",
    "    for comment in comments:\n",
    "        #user information\n",
    "        user_url = comment.select(\"div.comment_avatar > div > a\")[0][\"href\"]\n",
    "        user_id = user_url.split(\"/\")[-2]\n",
    "\n",
    "        #comment information\n",
    "        comment_id = comment[\"id\"].split(\"_\")[2]\n",
    "        floor = comment.select(\"div.comment_avatar > span\")[0].text\n",
    "        detail = comment.select(\"div.comment_conBox > div.comment_conWrap > div.comment_con > p > span\")[0].text\n",
    "        action = comment.select(\"div.comment_conBox > div.comment_conWrap > div.comment_action\")[0].text\n",
    "        published_date = comment.select(\"div.comment_conBox > div.comment_avatar_time > div.time > meta\")[0][\"content\"]\n",
    "        published_time = comment.select(\"div.comment_conBox > div.comment_avatar_time > div.time\")[0].text\n",
    "        action_dic=parse_action(action)\n",
    "\n",
    "        #format\n",
    "        comment_dic={\n",
    "          \"user_id\":user_id,\n",
    "          \"comment_id\":comment_id,\n",
    "          \"detail\":detail,\n",
    "          \"published_date\":published_date,\n",
    "          \"published_time\":published_time,\n",
    "          \"action\":action_dic ,\n",
    "        }\n",
    "        comments_dic[floor]=comment_dic\n",
    "        commenter_url_list.append(user_url)\n",
    "\n",
    "    return comments_dic,commenter_url_list\n",
    "\n",
    "def get_sigle_page_comments_structure(comments_structure,browser):\n",
    "    #click pad button\n",
    "    time.sleep(random.randint(0, 4))\n",
    "    pad_buttons=browser.find_elements(By.CSS_SELECTOR,\"div[id*=new_show_comment_more_div] > a\")\n",
    "    click_webelement(pad_buttons)\n",
    "    html=browser.page_source\n",
    "    doc_model = bf(html,'html.parser')\n",
    "\n",
    "    #parse\n",
    "    comments_block = doc_model.select(\"#commentTabBlockNew > ul.comment_listBox > li.comment_list\")\n",
    "\n",
    "    for i in range(len(comments_block)):\n",
    "        comment_block = comments_block[i]\n",
    "        floor = comment_block.span.text\n",
    "        comment_block_wraps = comment_block.select(\"div.comment_conBox > div.blockquote_wrap > blockquote\")\n",
    "        comment_block_dic = {}\n",
    "        for comment in comment_block_wraps:\n",
    "            comment_block_dic[int(comment.select(\".comment_floor\")[0].text)] = comment[\"blockquote_cid\"]\n",
    "        if comment_block_dic:\n",
    "            comments_structure[floor]=comment_block_dic\n",
    "\n",
    "    return comments_structure\n",
    "\n",
    "def turn_page_item(browser,page_num):\n",
    "    input_button = browser.find_element( By.CSS_SELECTOR , \"#commentTabBlockNew > ul.pagination > li.jumpToPage > input\")\n",
    "    input_button.click()\n",
    "    input_button.clear()\n",
    "    input_button.send_keys(page_num)\n",
    "    Go_button = browser.find_element(By.CSS_SELECTOR , \"#commentTabBlockNew > ul.pagination > li:last-child > a\")\n",
    "    Go_button.click()\n",
    "\n",
    "def get_comments(browser):\n",
    "    pages=get_pages(browser)\n",
    "    comments_dic = {}\n",
    "    comments_structure = {}\n",
    "    commenter_url_list = []\n",
    "    for i in range(pages):\n",
    "        page_index=i+1\n",
    "        if page_index==1:\n",
    "            html=browser.page_source\n",
    "            doc_model=bf(html,'html.parser')\n",
    "            comments_dic,commenter_url_list=get_sigle_page_comments_dict(comments_dic,doc_model,commenter_url_list)\n",
    "            comments_structure=get_sigle_page_comments_structure(comments_structure,browser)\n",
    "        else:\n",
    "            turn_page_item(browser,page_index)\n",
    "            html=browser.page_source\n",
    "            doc_model=bf(html,'html.parser')\n",
    "            comments_dic,commenter_url_list=get_sigle_page_comments_dict(comments_dic,doc_model,commenter_url_list)\n",
    "            comments_structure=get_sigle_page_comments_structure(comments_structure,browser)\n",
    "\n",
    "    comments = {\n",
    "        \"comments_detail\":comments_dic,\n",
    "        \"comments_structure\":comments_structure\n",
    "    }\n",
    "    return comments,list(set(commenter_url_list)) #strip identical url\n",
    "\n",
    "def get_article_labels_dict(doc_model):\n",
    "    article_labels = doc_model.select(\"#feed-main > div.item-name > div.artic-label-box > div.experience-meta-nowrap > ul > li > div > div\")\n",
    "    article_labels_dict = {}\n",
    "    label_index = 0\n",
    "    for i in range(len(article_labels)):\n",
    "        a = article_labels[i].text.split()[0].split(\"：\")\n",
    "        key = a[0]\n",
    "        value = a[1]\n",
    "        if key == '标签':\n",
    "            article_labels_dict[key+\"_\"+str(label_index)] = value\n",
    "            label_index+=1\n",
    "        else:\n",
    "            article_labels_dict[key+\"_\"+str(label_index)] = value\n",
    "    return  article_labels_dict\n",
    "\n",
    "def get_item_info(url):\n",
    "    browser=get_browser(url)\n",
    "    html=browser.page_source\n",
    "    doc_model = bf(html,'html.parser')\n",
    "    title = doc_model.select(\"#feed-main > div.info.J_info > div > div.title-box > h1\")[0].text\n",
    "    author_url = doc_model.select(\"#feed-main > div.info.J_info > div > div.info-details > div.author-info.J_author_info > a\")[0][\"href\"]\n",
    "    worthy_num = doc_model.select(\"#rating_worthy_num\")[0].text\n",
    "    unworthy_num = doc_model.select(\"#rating_unworthy_num\")[0].text\n",
    "    release_time = doc_model.select(\"#feed-main > div.info.J_info > div > div.info-details > div.author-info.J_author_info > span\")[0].text\n",
    "    article_labels_dict = get_article_labels_dict(doc_model)\n",
    "    introduction = doc_model.select(\"#feed-main > div.item-name > div.baike > div.introduce > div > div > p\")[0].text\n",
    "    item_image_url = doc_model.select(\"#feed-main > div.info.J_info > a > img\")[0][\"src\"]\n",
    "    price_info = doc_model.select(\"#feed-main > div.info.J_info > div > div.title-box > div > span > div\")[0].prettify()\n",
    "    feed_wrap = doc_model.select(\"#feed-wrap > div > a> span\")\n",
    "    class_hierarchy = ''\n",
    "    for i in feed_wrap:\n",
    "        class_hierarchy=class_hierarchy+i.text+'_'\n",
    "    comments , commenter_url_list = get_comments(browser)\n",
    "    crawl_time = time.strftime(\"%Y-%m-%d %H:%M:%S\", time.localtime())\n",
    "\n",
    "    item_info = {\n",
    "        \"title\" : title,\n",
    "        \"release_time\" : release_time,\n",
    "        \"crawl_time\" : crawl_time,\n",
    "        \"item_url\" : url,\n",
    "        \"item_image_url\" : item_image_url,\n",
    "        \"author_url\" : author_url,\n",
    "        \"worthy\" : {\"worthy_num\":worthy_num , \"unworthy_num\":unworthy_num},\n",
    "        \"article_labels\" : article_labels_dict,\n",
    "        \"introduction\" : introduction,\n",
    "        \"price_info\" : price_info,\n",
    "        \"class_hierarchy\" : class_hierarchy,\n",
    "        \"comments\" : comments,\n",
    "    }\n",
    "\n",
    "    graph_structure_item = {\n",
    "        \"creator_url\":author_url,\n",
    "        \"item_url\": url,\n",
    "        \"commenters_url\":commenter_url_list\n",
    "    }\n",
    "\n",
    "    return item_info,graph_structure_item\n",
    "\n",
    "def crawl_item(url):\n",
    "    try:\n",
    "\n",
    "        store_dir = os.path.join(store_path,\"item_{}\".format(url.split(\"/\")[-2]))\n",
    "        try:\n",
    "            os.mkdir(store_dir)\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "        item_info,graph_structure_item= get_item_info(url)\n",
    "        #print(item_info,graph_structure_item)\n",
    "        author_url, commenter_url_list = graph_structure_item[\"creator_url\"], graph_structure_item[\"commenters_url\"]\n",
    "\n",
    "        file_name_item_info = \"item_{}.json\".format(url.split(\"/\")[-2])\n",
    "        file_name_graph = \"item_{}_graph_structure.json\".format(url.split(\"/\")[-2])\n",
    "        write_json(file_name_item_info, item_info,store_dir)\n",
    "        write_json(file_name_graph, graph_structure_item,store_dir)\n",
    "\n",
    "        return author_url,commenter_url_list\n",
    "\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        return False\n",
    "    #url = 'https://www.smzdm.com/p/47801316/'\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    url = 'https://www.smzdm.com/p/49209839/'\n",
    "    author_url,commenter_url_list = crawl_item(url)\n",
    "    #print(author_url,commenter_url_list)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unable to click the button!\n",
      "unable to click the button!\n",
      "unable to click the button!\n",
      "unable to click the button!\n",
      "/Users/leslie/Desktop/item_info.json\n"
     ]
    }
   ],
   "source": [
    "url = 'https://www.smzdm.com/p/47801316/'\n",
    "item_info = get_item_info(url)\n",
    "write_json(\"item_info.json\",item_info)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "pycharm-69f4e5e5",
   "language": "python",
   "display_name": "PyCharm (local)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}